import sys

def loadBlackList():
    dict = {}
    inputFileHandler2 = open("/data/weijiang/BingDataSetDownloading/programs/url-static-update-black-list","r")
    for currentLine in inputFileHandler2.readlines():
        if currentLine.strip() not in dict:
            dict[currentLine.strip()] = 1
        else:
            dict[currentLine.strip()] += 1
    return dict



inputFileHandler = open("/data/weijiang/BingDataSetDownloading/urls-related/round07/urls-part00-OutOf17-hadoop01-round07","r")
blackListDict = loadBlackList()
counter = 0
for line in inputFileHandler.readlines():
    if line.strip() in blackListDict:
        print line.strip()
        counter += 1
    else:
        pass
        #print line.strip()
print counter,"Found in the urls list."

'''
inputFileName1 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/auxFiles/queryID_term_docID_pair_sorted_by_doc_id"
inputFileName2 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_term_docID_positions_features_partial.txt"


inputFileHandler2 = open(inputFileName2,"r")

numberOfLinesBeingComapred = len(inputFileHandler2.readlines())
inputFileHandler2.close()
print numberOfLinesBeingComapred

inputFileHandler2 = open(inputFileName2,"r")
inputFileHandler1 = open(inputFileName1,"r")
counter = 0

for currentLine in range(0,numberOfLinesBeingComapred):
    lineForFile1 = inputFileHandler1.readline()
    lineForFile2 = inputFileHandler2.readline()
    lineElementsForFile1 = lineForFile1.strip().split(" ")
    lineElementsForFile2 = lineForFile2.strip().split(" ")
    
    #if lineElementsForFile1[0] == lineElementsForFile2[0] and lineElementsForFile1[1] == lineElementsForFile2[1] and lineElementsForFile1[2] == lineElementsForFile2[2]:
    if lineElementsForFile1[2] == lineElementsForFile2[2]:
        counter += 1 
        pass
    else:
        counter += 1
        print "Problem"
        line1 = "*" + lineElementsForFile1[0] + " " +lineElementsForFile1[1] + " "+lineElementsForFile1[2] + "*"
        print "File1:" + line1
        
        
        line2 = "*" + lineElementsForFile2[0] + " " +lineElementsForFile2[1] + " "+lineElementsForFile2[2] + "*"
        print "File2:" + line2
        print "line:",counter
        sys.exit()

print "Pass the test. Good!"
'''

